﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Threading;
using System.Collections;

namespace Spider
{
    public class DocumentWorker
    {
        private Uri m_uri;
        private Spider m_spider;
        private Thread m_thread;
        private int m_number;
        public const string IndexFile = "index.html";
        static object objectlink = new object();
        private HttpHelper httpHelper;

        public DocumentWorker(Spider spider)
        {
            httpHelper = new HttpHelper();
            m_spider = spider;
        }
        #region 本地创建下载的目录结构
        private string convertFilename(Uri uri) //分离HTTP地址，创建相应的目录结构
        {
            string result = m_spider.OutputPath;
            int index1;
            int index2;

            if (result[result.Length - 1] != '\\')
                result = result + "\\";

            String path = uri.PathAndQuery;  //得到域名后的字符串

            int lastSlash = path.LastIndexOf('/');  //最后一个/
            int lastDot = path.LastIndexOf('.');    //最后一个.

            if (path[path.Length - 1] != '/')
            {
                if (lastSlash > lastDot)
                    path += "/" + IndexFile;
            }

            lastSlash = path.LastIndexOf('/');

            string filename = "";
            if (lastSlash != -1)
            {
                filename = path.Substring(1 + lastSlash);  //取得url后面的文件名及以后的
                path = path.Substring(0, 1 + lastSlash);  //取得文件名以前的
                filename = filename.Replace(".html", ".txt");
                if (filename.Equals(""))  //文件名为空的话，就设成index.html
                {
                    filename = IndexFile;
                }
            }
            // 必要时创建目录结构			
            index1 = 1;
            do
            {
                index2 = path.IndexOf('/', index1);   //获得目录的第二个/
                if (index2 != -1)
                {
                    String dirpart = path.Substring(index1, index2 - index1);
                    result += dirpart; //得到两个/之间的
                    result += "\\";

                    //try
                    //{
                    Directory.CreateDirectory(result);
                    //}
                    //catch
                    //{
                    //    return "";
                    //}
                    index1 = index2 + 1;
                }
            } while (index2 != -1);

            result += filename;
            Log.WriteLog(@Environment.CurrentDirectory + "\\url文件名.txt", result);
            return result;
        }
        #endregion

        #region 保存二进制文件
        private void SaveBinaryFile(WebResponse response) //存储二进制文件，二进制文件的内容类型声明不以"text/"开头
        {
            byte[] buffer = new byte[1024];
            if (m_spider.OutputPath == null)
            {
                return;
            }
            string filename = convertFilename(response.ResponseUri);

            if (filename.Contains("?"))
            {
                filename = filename.Replace("?", "").Replace(":", "");
            }
            if (filename == "")
                return;
            Stream outStream = File.Create(filename);  //确定了输出文件的名字和路径之后就可以打开读取Web页面的输入流、写入本地文件的输出流
            Stream inStream = response.GetResponseStream();

            int l;  //接下来就可以读取Web文件的内容并写入到本地文件，这可以通过一个循环方便地完成
            do
            {
                l = inStream.Read(buffer, 0, buffer.Length);
                if (l > 0)
                {
                    outStream.Write(buffer, 0, l);
                }
            }
            while (l > 0);
            outStream.Close();
            inStream.Close();
        }
        #endregion

        #region 保存文本文档
        private void SaveTextFile(string buffer) //文本文件的内容类型总是以"text/"开头，假设文件已被下载并保存到了一个字符串，这个字符串可以用来分析网页包含的链接，当然也可以保存为磁盘上的文件。     
        {
            #region 取标题，来做文件名
            string title = Utility.Resove(buffer, "<title>", "</title>");
            if (string.IsNullOrEmpty(title))
            {
                title = title.Replace(" ", "");
                title = title.Length > 15 ? title.Substring(0, 15) : title;
            }
            #endregion

            if (SpiderAccount.StartStr != string.Empty || SpiderAccount.EndStr != string.Empty)
            {
                buffer = Utility.Resove(buffer, SpiderAccount.StartStr, SpiderAccount.EndStr);
                if (string.IsNullOrEmpty(buffer))  //没有取到所要的字符串的话，就不保存
                    return;
            }
            buffer = buffer.Replace("&nbsp;", "");
            buffer = buffer.ToLower();

            buffer = System.Text.RegularExpressions.Regex.Replace(buffer, "<[^>]+>", "");//至少一个 
            buffer = System.Text.RegularExpressions.Regex.Replace(buffer, "<.+?>", "");
            //TempContent = System.Text.RegularExpressions.Regex.Replace(TempContent,"<[^>]*>","");任意个 

            if (m_spider.OutputPath == null)
            {
                return;
            }
            string filename = convertFilename(m_uri);
            if (filename.Contains("?"))
            {
                filename = filename.Split('?')[0];
            }
            title = Utility.ReplaceBadCharOfFileName(title);
            if (filename == "")
                return;
            else
            {
                string fileextension = filename.Substring(filename.LastIndexOf('.'));
                fileextension = ".txt";
                filename = filename.Substring(0, filename.LastIndexOf('.')) + title + fileextension;
            }

            StreamWriter outStream = new StreamWriter(filename, false, httpHelper.Encoding);//在这里，我们首先打开一个文件输出流，然后将字符串内容写入流，保存到文件里，最后关闭文件
            outStream.Write(buffer);
            outStream.Close();
        }
        #endregion

        #region 下载一个url
        private string GetPage()  //下载一个文本文档或是二进制文件（图片）
        {
            HttpWebResponse response = null;
            Stream stream = null;
            StreamReader reader = null;
            try
            {
                m_spider.ShowUrl(m_uri.ToString(), false);
                response = httpHelper.GetHtmlResponse(m_uri.ToString(), CookieMng.CurrentCookie); //HttpWebRequest类是来下载HTML页面
                stream = response.GetResponseStream();

                if (!response.ContentType.ToLower().StartsWith("text/"))  //确定该文件是否为二进制文件
                {
                    SaveBinaryFile(response);
                    return null;  //不要图片，直接返回
                }
                string buffer = "", line;
                reader = new StreamReader(stream, httpHelper.Encoding); //如果该文件是文本文件，首先从Stream创建一个StreamReader，然后将文本文件的内容一行一行读取出来
                while ((line = reader.ReadLine()) != null)
                {
                    buffer += line + "\r\n";
                }
                lock (objectlink)
                {
                    m_spider.m_urlCount++;
                }
                //buffer = reader.ReadToEnd();
                SaveTextFile(buffer);
                return buffer;
            }
            catch (WebException e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                System.Console.WriteLine("下载失败，错误：" + e);
                //throw e;
                Log.WriteLog(e.Message);
                m_spider.ShowUrl(m_uri.ToString(), true);
                return null;
            }
            catch (IOException e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                System.Console.WriteLine("下载失败，错误：" + e);
                //throw e;
                Log.WriteLog(e.Message);
                m_spider.ShowUrl(m_uri.ToString(), true);
                return null;
            }
            catch (Exception e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                Log.WriteLog(e.Message);
                m_spider.ShowUrl(m_uri.ToString(), true);
                return null;
            }
            finally
            {
                if (reader != null) reader.Close();
                if (stream != null) stream.Close();
                if (response != null) response.Close();
            }
        }
        #endregion

        #region 把解析后的url加入到队列中
        private void ProcessLink(string link)
        {
            Uri url;
            try
            {
                m_uri = new Uri(link);
                if (m_uri.Host != SpiderAccount.UrlHost)
                {
                    #region 验证非本域名下的URL
                    int k = link.LastIndexOf('.');
                    string outurl = link.Substring(k + 1);  //取后缀名
                    if (!SpiderAccount.AllowURLType.Contains(outurl))
                    {
                        Log.LogSpecial(LogDetail.LogType.AllOutUrl, m_uri.ToString());
                        return;
                    }
                    #endregion
                }
                else
                {
                    #region 下载指定目录下的页面
                    int k = link.LastIndexOf('.');
                    string outurl = link.Substring(k + 1);  //取后缀名
                    if (!SpiderAccount.AllowURLType.Contains(outurl))
                    {
                        string contain = SpiderAccount.UrlContain;
                        if (!string.IsNullOrEmpty(contain))
                        {
                            if (!link.Contains(contain))
                            {
                                return;
                            }
                        }
                    }
                    #endregion
                }
                url = m_uri;
            }
            catch (UriFormatException e)
            {
                //System.Console.WriteLine("Invalid URI:" + link + " Error:" + e.Message);
                //return;
                Log.WriteLog(e.Message);
                return;
            }
            if (!url.Scheme.ToLower().Equals("http") && !url.Scheme.ToLower().Equals("https"))
            {
                return;
            }
            if (!url.Host.ToLower().Equals(m_uri.Host.ToLower()))  //一个host下进行搜索
            {
                return;
            }
            m_spider.addURI(url);  //加入到队列中，是否重复会自动判断
        }
        #endregion

        #region 处理已经下载过的text类型的文档
        private void ProcessPage(string page, Uri uri)
        {
            string content = page;
            //IList<string> list = ParseHtmlNew.GetHyperLink(page);
            IList<string> list = ParseHtmlNew.GetHyperLinkByParse(page);
            foreach (string s in list)
            {
                string link = s;
                HandleSrcAndHref(ref link, uri);
                ProcessLink(link);
            }
        }
        #endregion

        #region  处理src或href上面的URL
        public void HandleSrcAndHref(ref string link, Uri uri)
        {
            lock (objectlink)
            {
                Log.WriteLog(@Environment.CurrentDirectory + "\\Url处理.txt", link + "----------" + uri.ToString());
                string url = uri.ToString();
                url = url.Remove(url.LastIndexOf("/"));   //移除后面的文件名
                if (link.IndexOf('/') == 0)  //相对根目录的路径
                {
                    link = SpiderAccount.Url + link;
                }
                else if (link.Contains(".."))  //如果链接中含有..
                {
                    int count = 0; //i表示的是..的数量
                    int i = link.Length;
                    do
                    {
                        int j = link.LastIndexOf("..", i);
                        if (j >= 0)
                        {
                            count++;
                            link = link.Remove(j, 2);   //移除..
                            url = url.Remove(url.LastIndexOf("/"));

                            i = j;
                        }
                    }
                    while (i > 0);

                    link = url + link;
                }
                else if (!link.Contains("http:"))
                {
                    link = url.ToString() + "/" + link;
                }
                //Log.WriteLog(link);
            }
        }
        #endregion

        #region 获取一个可用的url，并下载它，不可用的话，将一直等待（ObtainWork的作用）
        public void Process()
        //多线程使得计算机看起来就像能够同时执行一个以上的操作，不过，除非计算机包含多个处理器，否则，所谓的同时执行多个
        //操作仅仅是一种模拟出来的效果。靠计算机在多个线程之间快速切换达到”同时“执行多个操作的效果。一般而言，只有在两种
        //情况下才能事实上提高程序运行的速度。
        //第一种情况使计算机拥有多个处理器，第二种情况是程序经常要等待某个外部事件。
        //为此，我们用DocumentWorker类封装所有下载一个URL的操作。每当一个DocumentWorker的实例被创建，它就进入循环，等待下一个要处理的URL。
        {
            while (!m_spider.Quit) //这个循环将一直运行，直至Quit标记被设置成了true（当用户点击"Cancel"按钮时，Quit标记就被设置成true）。
            {
                try
                {
                    m_uri = m_spider.ObtainWork();  //获取一个URL，ObtainWork将一直等待，直到有一个URL可有，这要有其它线程解析文档并寻找链接才能获得

                    m_spider.SpiderDone.WorkerBegin();
                    string num = "Download(" + this.Number + "):" + m_uri;

                    Log.WriteLog(num); //记下线程下载的url
                    string page = GetPage();  //得到的是一个字符串形式的html文档
                    if (page != null)
                    {
                        ProcessPage(page, m_uri);  //传递一个本页面的url，可以帮助正确处理url路径问题
                    }
                    m_spider.SpiderDone.WorkerEnd();
                }
                catch (Exception err)
                {
                    System.Windows.Forms.MessageBox.Show(err.Message);
                }
            }
        }
        #endregion

        public void start()  //开始处理url
        {
            ThreadStart ts = new ThreadStart(this.Process);
            m_thread = new Thread(ts);
            m_thread.IsBackground = true;  //设置后台线程
            m_thread.Start();
        }
        public int Number  //用来标识线程的
        {
            get { return m_number; }
            set { m_number = value; }
        }
    }
}
